{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/gym/core.py:26: UserWarning: \u001b[33mWARN: Gym minimally supports python 3.6 as the python foundation not longer supports the version, please update your version to 3.7+\u001b[0m\n",
      "  \"Gym minimally supports python 3.6 as the python foundation not longer supports the version, please update your version to 3.7+\"\n",
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/gym/core.py:330: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
      "  \"Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\"\n",
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/gym/wrappers/step_api_compatibility.py:40: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
      "  \"Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\"\n",
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/gym/core.py:52: DeprecationWarning: \u001b[33mWARN: The argument mode in render method is deprecated; use render_mode during environment initialization instead.\n",
      "See here for more information: https://www.gymlibrary.ml/content/api/\u001b[0m\n",
      "  \"The argument mode in render method is deprecated; \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAD8CAYAAAB3lxGOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQj0lEQVR4nO3df2jc933H8edLv+1YSWxLVmRJjpRWhSVN5gbhdWSMknbE/UFtAgEXOvxHIBQ8mrJBZ6+0pbBAtz9K/1jzR2jLDO1qXNIQE8o246YNZaOunKRZbNe1GtuSYkWSkzp2bP063Xt/3Dfu1Zati3WnO+XzeoC47330vdP7wHrq+707S4oIzCxdddUewMyqyxEwS5wjYJY4R8AscY6AWeIcAbPEVSwCkrZKOiFpSNLuSn0dM1saVeJ9ApLqgd8BfwOMAr8GPhcRx8r+xcxsSSp1JLAFGIqI1yJiFtgHbKvQ1zKzJWio0P12ASNF10eBv7jezm1tbdHb21uhUcwM4MiRI+ciov3q9UpFQAus/cl5h6THgMcANm3axODgYIVGMTMASWcWWq/U6cAo0FN0vRs4W7xDRDwVEQMRMdDefk2czGyZVCoCvwb6JfVJagJ2AAcq9LXMbAkqcjoQETlJfwf8F1APfD8ijlbia5nZ0lTqOQEi4qfATyt1/2ZWHn7HoFniHAGzxDkCZolzBMwS5wiYJc4RMEucI2CWOEfALHGOgFniHAGzxDkCZolzBMwS5wiYJc4RMEucI2CWOEfALHGOgFniHAGzxDkCZolzBMwS5wiYJc4RMEucI2CWOEfALHGOgFniHAGzxDkCZolzBMwS5wiYJc4RMEucI2CWOEfALHGOgFniHAGzxDkCZolbNAKSvi9pQtKrRWvrJB2UdDK7XFv0uT2ShiSdkPRQpQY3s/Io5Ujg34GtV63tBg5FRD9wKLuOpLuBHcA92W2elFRftmnNrOwWjUBEvAC8ddXyNmBvtr0X2F60vi8iZiLiFDAEbCnPqGZWCTf7nEBHRIwBZJcbsvUuYKRov9Fs7RqSHpM0KGlwcnLyJscws6Uq9xODWmAtFtoxIp6KiIGIGGhvby/zGGZWqpuNwLikToDsciJbHwV6ivbrBs7e/HhmVmk3G4EDwM5seyfwbNH6DknNkvqAfuDw0kY0s0pqWGwHST8CPga0SRoFvg58E9gv6VFgGHgEICKOStoPHANywK6ImK/Q7GZWBotGICI+d51Pffw6+z8BPLGUocxs+fgdg2aJcwTMEucImCXOETBLnCNgljhHwCxxjoBZ4hwBs8Q5AmaJcwTMEucImCXOETBLnCNgljhHwCxxjoBZ4hwBs8Q5AmaJcwTMEucImCXOETBLnCNgljhHwCxxjoBZ4hwBs8Q5AmaJcwTMEucImCXOETBLnCNgljhHwCxxjoBZ4hwBs8Q5AmaJcwTMErdoBCT1SHpe0nFJRyU9nq2vk3RQ0snscm3RbfZIGpJ0QtJDlXwAZrY0pRwJ5IB/iIg/Az4K7JJ0N7AbOBQR/cCh7DrZ53YA9wBbgScl1VdieDNbukUjEBFjEfFitn0ROA50AduAvdlue4Ht2fY2YF9EzETEKWAI2FLmuc2sTN7TcwKSeoGPAL8COiJiDAqhADZku3UBI0U3G83WzKwGlRwBSWuAp4EvRcSFG+26wFoscH+PSRqUNDg5OVnqGGZWZiVFQFIjhQD8MCJ+ki2PS+rMPt8JTGTro0BP0c27gbNX32dEPBURAxEx0N7efrPzm9kSlfLqgIDvAccj4ltFnzoA7My2dwLPFq3vkNQsqQ/oBw6Xb2QzK6eGEvZ5APhb4P8kvZyt/RPwTWC/pEeBYeARgIg4Kmk/cIzCKwu7ImK+3IObWXksGoGI+CULn+cDfPw6t3kCeGIJc5nZMvE7Bs0SV8rpgNn7QuTzRC5H5HKFhbo66pqaUF3aPwsdAXtfinye/PQ085cvMzs5yfTICFMjI0yPjjI9OkrMz9PQ2sqtmzfTtnUrzXfcQeE58PQ4ArYiRRS99SSfJ3fxInNvv83sG28wdfo0U2fOMDMxwezEBPOXLhHz85DP/8l9zJ07x9SpU7z94ov0fvGLrP7gB5MMgSNgK0JEwPw8+VyOuT/8gdnxcabPnmV6eJip4WHm3nyTufPnyU9Nvef7nj5zhjPf+Q79X/86jWvXLn6D9xlHwGpO5PPkZ2fJT08XDuVff/3KYfz06Ci5CxcKP93n5sr2Nadee423XniBjm3bynafK4UjYFUV8/PMX7pE7sIFZsbHmRoeZnpkhJmxMWbeeIP5d94hn8vBfOXfanL5tdcq/jVqkSNgFRcREEHkcsydP8/cuXNMj40xdfo00yMjzE5OMnvuHPmZmWvO263yHAErmyvf7HNz5GdnmT13jpmxMaZff73wE350lNz58+QuXiRmZ6s97jWaOzurPUJVOAJ202J+nvmpKeYvXWJmfJzpd1+Cy87h85cvk5+d/ePr8jWsubOT9Q8+WO0xqsIRsJJFBHNvvcU7x44xdeYMU8PDzI6PMzs5SX56uvAyXFzzv8ZrXmNbG5u+8AWaEv3frI6AlSQ/O8ubzz/P+DPPMHP2mv8ZvrJIqL6e+tWrad28mY7t21n9gQ8k+R4BcASsBPlcjjeeeYY3fvzjmjyXX0xdczN1q1bR1N7Oqp4eWnp6WHXnnbT09NC4bh11jY3VHrGqHAG7oYjg4iuvMP7008TsLBdmZ/nJ8DCT09M8tHEj965dW1M/Qeuam2m4/Xaa1q9nVV8fq3p7adm4kebOTurXrCl8w9fV1dTM1eYI2KImn3uO/PQ0F+fm+NpLL/HLicIvkfrP0VH++f77+csNGxa5hwqoq6OuuZmmtjaaOjpYdeedrO7ro7mzk6a2NhpuvRWy/xjkb/gbcwTsxiKYe/ttAF6/fJn/mZi48qm35+b477NnKxsBCTU2Fr7h29tp6eqipbub1XfdRUtXF/W33krDLbf4p/sSOAJWsqa6Oprr65kqevferWU+n65rbqa+tbVwON/bWzh37+6meeNGGtasoa6pCerr/Q1fRo6A3ZhE6333cfnkSfrWrOEf772Xfzt+nIu5HA9s2MCj/f03fb91TU00rl9feMKut/fK+XtTRwcNra2ovr5wJOBv+IpyBOyGJNH+yU9y4cgRpk6f5tPd3dy/fj1TuRxdt9xCS/0if1wqeznu3cP55jvuoGXTpiuH8w233UZDa6vP36vIEbBFNbW3s2nXLoaffJKp06fZuHr1dfe98nJcW9uVl+FWbdpEc1cXDa2t1LW0IB/O1xRHwBYliVs+9CE++NWv8tYvfsHFV15h7vx5ZsbGaLjttivf8Kt6e2np6qK5s7NwON/Q4CfsVgBHwEoiiaa2NjoefpiOhx8mPzPDzNmzNK5d65fjVjhHwN6Td7/J61taWH3XXVWexsoh7V+zamaOgFnqHAGzxDkCZolzBMwS5wiYJc4RMEucI2CWOEfALHGOgFniHAGzxDkCZolbNAKSWiQdlvQbSUclfSNbXyfpoKST2eXaotvskTQk6YSkhyr5AMxsaUo5EpgBHoyIPwc2A1slfRTYDRyKiH7gUHYdSXcDO4B7gK3Ak5IW+fUzZlYti0YgCt7JrjZmHwFsA/Zm63uB7dn2NmBfRMxExClgCNhSzqHNrHxKek5AUr2kl4EJ4GBE/AroiIgxgOzy3d873QWMFN18NFu7+j4fkzQoaXBycnIJD8HMlqKkCETEfERsBrqBLZI+fIPdF/rVMtf8lcqIeCoiBiJioD3RPwRpVgve06sDEXEe+DmFc/1xSZ0A2eW7f5ViFOgpulk3sML/gqXZ+1cprw60S7o9214FfAL4LXAA2JntthN4Nts+AOyQ1CypD+gHDpd5bjMrk1J+x2AnsDd7hr8O2B8Rz0n6X2C/pEeBYeARgIg4Kmk/cAzIAbsiYv46921mVaaIa07Xl93AwEAMDg5Wewyz9zVJRyJi4Op1v2PQLHGOgFniHAGzxDkCZolzBMwS5wiYJc4RMEucI2CWOEfALHGOgFniHAGzxDkCZolzBMwS5wiYJc4RMEucI2CWOEfALHGOgFniHAGzxDkCZolzBMwS5wiYJc4RMEucI2CWOEfALHGOgFniHAGzxDkCZolzBMwS5wiYJc4RMEucI2CWOEfALHGOgFniSo6ApHpJL0l6Lru+TtJBSSezy7VF++6RNCTphKSHKjG4mZXHezkSeBw4XnR9N3AoIvqBQ9l1JN0N7ADuAbYCT0qqL8+4ZlZuJUVAUjfwaeC7RcvbgL3Z9l5ge9H6voiYiYhTwBCwpSzTmlnZlXok8G3gy0C+aK0jIsYAsssN2XoXMFK032i2ZmY1aNEISPoMMBERR0q8Ty2wFgvc72OSBiUNTk5OlnjXZlZupRwJPAB8VtJpYB/woKQfAOOSOgGyy4ls/1Ggp+j23cDZq+80Ip6KiIGIGGhvb1/CQzCzpVg0AhGxJyK6I6KXwhN+P4uIzwMHgJ3ZbjuBZ7PtA8AOSc2S+oB+4HDZJzezsmhYwm2/CeyX9CgwDDwCEBFHJe0HjgE5YFdEzC95UjOrCEVcc7q+7AYGBmJwcLDaY5i9r0k6EhEDV6/7HYNmiXMEzBLnCJglzhEwS5wjYJY4R8AscY6AWeIcAbPEOQJmiXMEzBLnCJglzhEwS5wjYJY4R8AscY6AWeIcAbPEOQJmiXMEzBLnCJglzhEwS5wjYJY4R8AscY6AWeIcAbPEOQJmiXMEzBLnCJglzhEwS5wjYJY4R8AscY6AWeIcAbPEOQJmiXMEzBLnCJglzhEwS5wjYJY4R8AscYqIas+ApEngEnCu2rOUqI2VMyusrHk9a+XcGRHtVy/WRAQAJA1GxEC15yjFSpoVVta8nnX5+XTALHGOgFniaikCT1V7gPdgJc0KK2tez7rMauY5ATOrjlo6EjCzKqh6BCRtlXRC0pCk3dWeB0DS9yVNSHq1aG2dpIOSTmaXa4s+tyeb/4Skh5Z51h5Jz0s6LumopMdrdV5JLZIOS/pNNus3anXWoq9fL+klSc/V+qw3LSKq9gHUA78H7gKagN8Ad1dzpmyuvwbuB14tWvtXYHe2vRv4l2z77mzuZqAvezz1yzhrJ3B/tt0K/C6bqebmBQSsybYbgV8BH63FWYtm/nvgP4DnavnfwVI+qn0ksAUYiojXImIW2Adsq/JMRMQLwFtXLW8D9mbbe4HtRev7ImImIk4BQxQe17KIiLGIeDHbvggcB7pqcd4oeCe72ph9RC3OCiCpG/g08N2i5ZqcdSmqHYEuYKTo+mi2Vos6ImIMCt94wIZsvWYeg6Re4CMUfsLW5LzZ4fXLwARwMCJqdlbg28CXgXzRWq3OetOqHQEtsLbSXq6oiccgaQ3wNPCliLhwo10XWFu2eSNiPiI2A93AFkkfvsHuVZtV0meAiYg4UupNFlhbEf+Wqx2BUaCn6Ho3cLZKsyxmXFInQHY5ka1X/TFIaqQQgB9GxE+y5ZqdFyAizgM/B7ZSm7M+AHxW0mkKp6kPSvpBjc66JNWOwK+Bfkl9kpqAHcCBKs90PQeAndn2TuDZovUdkpol9QH9wOHlGkqSgO8BxyPiW7U8r6R2Sbdn26uATwC/rcVZI2JPRHRHRC+Ff5c/i4jP1+KsS1btZyaBT1F4Rvv3wFeqPU8204+AMWCOQuEfBdYDh4CT2eW6ov2/ks1/AvjkMs/6VxQOO18BXs4+PlWL8wL3AS9ls74KfC1br7lZr5r7Y/zx1YGanvVmPvyOQbPEVft0wMyqzBEwS5wjYJY4R8AscY6AWeIcAbPEOQJmiXMEzBL3/zOczWLFvL9aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "#创建环境\n",
    "env = gym.make('Pendulum-v1')\n",
    "env.reset()\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "这个游戏的状态用3个数字表示,我也不知道这3个数字分别是什么意思,反正这3个数字就能描述游戏全部的状态\n",
      "state= [0.34507838 0.9385739  0.21112834]\n",
      "这个游戏的动作是个-2到+2之间的连续值\n",
      "env.action_space= Box(-2.0, 2.0, (1,), float32)\n",
      "随机一个动作\n",
      "action= [1.3828996]\n",
      "执行一个动作,得到下一个状态,奖励,是否结束\n",
      "state= [0.29188552 0.95645326 1.1224937 ]\n",
      "reward= -1.4910488871776209\n",
      "over= False\n"
     ]
    }
   ],
   "source": [
    "#测试游戏环境\n",
    "def test_env():\n",
    "    state = env.reset()\n",
    "    print('这个游戏的状态用3个数字表示,我也不知道这3个数字分别是什么意思,反正这3个数字就能描述游戏全部的状态')\n",
    "    print('state=', state)\n",
    "    #state= [-0.91304934 -0.40784913  0.271098  ]\n",
    "\n",
    "    print('这个游戏的动作是个-2到+2之间的连续值')\n",
    "    print('env.action_space=', env.action_space)\n",
    "    #env.action_space= Box(-2.0, 2.0, (1,), float32)\n",
    "\n",
    "    print('随机一个动作')\n",
    "    action = env.action_space.sample()\n",
    "    print('action=', action)\n",
    "    #action= [-0.14946985]\n",
    "\n",
    "    print('执行一个动作,得到下一个状态,奖励,是否结束')\n",
    "    state, reward, over, _ = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    #state= [-0.5629868  0.8264659  2.7232552]\n",
    "\n",
    "    print('reward=', reward)\n",
    "    #reward= -4.456876123969679\n",
    "\n",
    "    print('over=', over)\n",
    "    #over= False\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Sequential(\n",
       "   (0): Linear(in_features=3, out_features=128, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=128, out_features=11, bias=True)\n",
       " ),\n",
       " Sequential(\n",
       "   (0): Linear(in_features=3, out_features=128, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=128, out_features=11, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#计算动作的模型,也是真正要用的模型\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(3, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 11),\n",
    ")\n",
    "\n",
    "#经验网络,用于评估一个状态的分数\n",
    "next_model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(3, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 11),\n",
    ")\n",
    "\n",
    "#把model的参数复制给next_model\n",
    "next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "model, next_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7, 0.7999999999999998)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "def get_action(state):\n",
    "    #走神经网络,得到一个动作\n",
    "    state = torch.FloatTensor(state).reshape(1, 3)\n",
    "    action = model(state).argmax().item()\n",
    "\n",
    "    if random.random() < 0.01:\n",
    "        action = random.choice(range(11))\n",
    "\n",
    "    #离散动作连续化\n",
    "    action_continuous = action\n",
    "    action_continuous /= 10\n",
    "    action_continuous *= 4\n",
    "    action_continuous -= 2\n",
    "\n",
    "    return action, action_continuous\n",
    "\n",
    "\n",
    "get_action([0.29292667, 0.9561349, 1.0957013])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((200, 0), 200)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#样本池\n",
    "datas = []\n",
    "\n",
    "\n",
    "#向样本池中添加N条数据,删除M条最古老的数据\n",
    "def update_data():\n",
    "    old_count = len(datas)\n",
    "\n",
    "    #玩到新增了N个数据为止\n",
    "    while len(datas) - old_count < 200:\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action, action_continuous = get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            next_state, reward, over, _ = env.step([action_continuous])\n",
    "\n",
    "            #记录数据样本\n",
    "            datas.append((state, action, reward, next_state, over))\n",
    "\n",
    "            #更新游戏状态,开始下一个动作\n",
    "            state = next_state\n",
    "\n",
    "    update_count = len(datas) - old_count\n",
    "    drop_count = max(len(datas) - 5000, 0)\n",
    "\n",
    "    #数据上限,超出时从最古老的开始删除\n",
    "    while len(datas) > 5000:\n",
    "        datas.pop(0)\n",
    "\n",
    "    return update_count, drop_count\n",
    "\n",
    "\n",
    "update_data(), len(datas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/ipykernel_launcher.py:7: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  /opt/conda/conda-bld/pytorch_1640811701593/work/torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[-9.9947e-01,  3.2551e-02, -3.8519e-01],\n",
       "         [-9.9985e-01,  1.7046e-02,  2.1124e-01],\n",
       "         [-9.9987e-01,  1.5936e-02,  2.0854e-01],\n",
       "         [-9.9984e-01,  1.8116e-02,  1.8792e-01],\n",
       "         [-9.9996e-01,  8.7692e-03, -4.0722e-01],\n",
       "         [-9.9832e-01,  5.7964e-02, -4.2006e-02],\n",
       "         [-9.9931e-01, -3.7169e-02,  2.5256e-01],\n",
       "         [-9.9784e-01,  6.5725e-02, -6.9845e-02],\n",
       "         [-9.9881e-01, -4.8671e-02, -7.2914e-02],\n",
       "         [-9.9991e-01,  1.3297e-02, -3.3516e-01],\n",
       "         [-1.0000e+00,  9.4035e-04, -4.4795e-01],\n",
       "         [-9.9848e-01,  5.5079e-02,  2.2331e-01],\n",
       "         [-9.9962e-01,  2.7509e-02,  1.6729e-01],\n",
       "         [-9.9873e-01,  5.0328e-02,  9.1262e-02],\n",
       "         [-9.9865e-01,  5.1885e-02,  3.3095e-02],\n",
       "         [-9.9879e-01,  4.9143e-02, -4.5993e-02],\n",
       "         [-9.9970e-01, -2.4547e-02,  2.7097e-01],\n",
       "         [-9.9681e-01, -7.9855e-02,  2.0785e-02],\n",
       "         [-9.9650e-01,  8.3577e-02, -5.7595e-03],\n",
       "         [-9.9827e-01,  5.8763e-02, -4.0141e-02],\n",
       "         [-9.9857e-01,  5.3538e-02, -7.0582e-03],\n",
       "         [-9.9873e-01,  5.0462e-02, -2.1796e-01],\n",
       "         [-9.9993e-01, -1.1591e-02, -3.3853e-01],\n",
       "         [-9.9806e-01,  6.2240e-02, -1.1652e-01],\n",
       "         [-9.9995e-01, -1.0241e-02, -3.4000e-01],\n",
       "         [-9.9951e-01, -3.1411e-02,  1.7949e-01],\n",
       "         [-9.9781e-01,  6.6217e-02, -1.3552e-01],\n",
       "         [-9.9776e-01, -6.6838e-02, -1.0927e-01],\n",
       "         [-9.9773e-01,  6.7378e-02,  7.4442e-02],\n",
       "         [-9.9874e-01, -5.0175e-02,  2.7805e-01],\n",
       "         [-9.9932e-01, -3.6916e-02,  1.9112e-01],\n",
       "         [-9.9823e-01,  5.9454e-02, -1.8011e-01],\n",
       "         [-9.9884e-01,  4.8198e-02,  2.8064e-02],\n",
       "         [-9.9868e-01,  5.1325e-02,  8.6622e-02],\n",
       "         [-9.9978e-01, -2.1029e-02,  3.2103e-01],\n",
       "         [-9.9844e-01,  5.5867e-02, -8.3907e-02],\n",
       "         [-9.9719e-01,  7.4885e-02,  1.1748e-01],\n",
       "         [-9.9936e-01,  3.5795e-02,  1.6192e-01],\n",
       "         [-9.9843e-01,  5.5929e-02, -7.8190e-02],\n",
       "         [-9.9963e-01, -2.7366e-02,  2.1164e-01],\n",
       "         [-9.9787e-01,  6.5278e-02,  2.9511e-02],\n",
       "         [-9.9898e-01, -4.5081e-02,  1.6343e-01],\n",
       "         [-9.9935e-01,  3.5998e-02,  2.4421e-01],\n",
       "         [-9.9890e-01,  4.6846e-02, -8.1128e-02],\n",
       "         [-9.9730e-01, -7.3406e-02, -2.7688e-01],\n",
       "         [-9.9997e-01,  8.0432e-03,  2.8804e-01],\n",
       "         [-9.9890e-01,  4.6866e-02,  1.6585e-01],\n",
       "         [-9.9881e-01,  4.8814e-02, -3.8998e-02],\n",
       "         [-9.9858e-01,  5.3185e-02, -4.6947e-02],\n",
       "         [-9.9986e-01,  1.6810e-02,  2.6440e-01],\n",
       "         [-9.9995e-01, -1.0009e-02, -1.8566e-01],\n",
       "         [-9.9793e-01,  6.4360e-02, -2.3154e-01],\n",
       "         [-9.9751e-01,  7.0500e-02, -8.5860e-02],\n",
       "         [-9.9883e-01,  4.8289e-02,  7.2009e-02],\n",
       "         [-9.9891e-01,  4.6592e-02, -1.2002e-01],\n",
       "         [-9.9925e-01,  3.8681e-02, -1.8609e-01],\n",
       "         [-9.9855e-01, -5.3850e-02, -3.9193e-01],\n",
       "         [-9.9904e-01,  4.3737e-02,  6.9727e-02],\n",
       "         [-9.9992e-01, -1.2625e-02,  2.0579e-01],\n",
       "         [-9.9988e-01, -1.5546e-02, -2.7426e-01],\n",
       "         [-9.9740e-01, -7.2041e-02,  1.1898e-01],\n",
       "         [-9.9971e-01,  2.4239e-02, -2.5877e-01],\n",
       "         [-9.9781e-01, -6.6106e-02,  1.6856e-01],\n",
       "         [-9.8685e-01, -1.6165e-01, -1.0353e-01]]),\n",
       " tensor([[9],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [9],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [4],\n",
       "         [4],\n",
       "         [9],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [4],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [4],\n",
       "         [5],\n",
       "         [4],\n",
       "         [5],\n",
       "         [5],\n",
       "         [4],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [8],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [4],\n",
       "         [5],\n",
       "         [1],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [4],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [5],\n",
       "         [4],\n",
       "         [5],\n",
       "         [5],\n",
       "         [4],\n",
       "         [4],\n",
       "         [4],\n",
       "         [5],\n",
       "         [4]]),\n",
       " tensor([[-9.6835],\n",
       "         [-9.7673],\n",
       "         [-9.7741],\n",
       "         [-9.7596],\n",
       "         [-9.8337],\n",
       "         [-9.5087],\n",
       "         [-9.6438],\n",
       "         [-9.4612],\n",
       "         [-9.5667],\n",
       "         [-9.7976],\n",
       "         [-9.8863],\n",
       "         [-9.5314],\n",
       "         [-9.7003],\n",
       "         [-9.5566],\n",
       "         [-9.5463],\n",
       "         [-9.5633],\n",
       "         [-9.7233],\n",
       "         [-9.3739],\n",
       "         [-9.3509],\n",
       "         [-9.5038],\n",
       "         [-9.5359],\n",
       "         [-9.5597],\n",
       "         [-9.8085],\n",
       "         [-9.4835],\n",
       "         [-9.8171],\n",
       "         [-9.6764],\n",
       "         [-9.4595],\n",
       "         [-9.4552],\n",
       "         [-9.4510],\n",
       "         [-9.5645],\n",
       "         [-9.6426],\n",
       "         [-9.5026],\n",
       "         [-9.5705],\n",
       "         [-9.5504],\n",
       "         [-9.7482],\n",
       "         [-9.5222],\n",
       "         [-9.4056],\n",
       "         [-9.6486],\n",
       "         [-9.5218],\n",
       "         [-9.7029],\n",
       "         [-9.4635],\n",
       "         [-9.5910],\n",
       "         [-9.6506],\n",
       "         [-9.5780],\n",
       "         [-9.4212],\n",
       "         [-9.8274],\n",
       "         [-9.5825],\n",
       "         [-9.5653],\n",
       "         [-9.5383],\n",
       "         [-9.7713],\n",
       "         [-9.8104],\n",
       "         [-9.4744],\n",
       "         [-9.4320],\n",
       "         [-9.5689],\n",
       "         [-9.5804],\n",
       "         [-9.6315],\n",
       "         [-9.5495],\n",
       "         [-9.5971],\n",
       "         [-9.7947],\n",
       "         [-9.7798],\n",
       "         [-9.4233],\n",
       "         [-9.7247],\n",
       "         [-9.4612],\n",
       "         [-8.8770]]),\n",
       " tensor([[-0.9993,  0.0386, -0.1208],\n",
       "         [-1.0000,  0.0058,  0.2240],\n",
       "         [-1.0000,  0.0049,  0.2205],\n",
       "         [-1.0000,  0.0080,  0.2015],\n",
       "         [-0.9999,  0.0168, -0.1606],\n",
       "         [-0.9983,  0.0579,  0.0015],\n",
       "         [-0.9988, -0.0484,  0.2247],\n",
       "         [-0.9978,  0.0667, -0.0206],\n",
       "         [-0.9992, -0.0402, -0.1694],\n",
       "         [-0.9995,  0.0326, -0.3852],\n",
       "         [-0.9999,  0.0113, -0.2072],\n",
       "         [-0.9991,  0.0419,  0.2646],\n",
       "         [-0.9998,  0.0181,  0.1879],\n",
       "         [-0.9990,  0.0439,  0.1290],\n",
       "         [-0.9988,  0.0483,  0.0720],\n",
       "         [-0.9988,  0.0496, -0.0091],\n",
       "         [-0.9993, -0.0372,  0.2526],\n",
       "         [-0.9972, -0.0749, -0.0991],\n",
       "         [-0.9967,  0.0807,  0.0569],\n",
       "         [-0.9983,  0.0586,  0.0039],\n",
       "         [-0.9987,  0.0519,  0.0331],\n",
       "         [-0.9982,  0.0595, -0.1801],\n",
       "         [-1.0000,  0.0088, -0.4072],\n",
       "         [-0.9978,  0.0657, -0.0698],\n",
       "         [-0.9999,  0.0101, -0.4077],\n",
       "         [-0.9992, -0.0392,  0.1559],\n",
       "         [-0.9975,  0.0705, -0.0859],\n",
       "         [-0.9984, -0.0559, -0.2194],\n",
       "         [-0.9981,  0.0611,  0.1250],\n",
       "         [-0.9981, -0.0622,  0.2404],\n",
       "         [-0.9990, -0.0451,  0.1634],\n",
       "         [-0.9978,  0.0662, -0.1355],\n",
       "         [-0.9994,  0.0360,  0.2442],\n",
       "         [-0.9990,  0.0451,  0.1251],\n",
       "         [-0.9993, -0.0363,  0.3053],\n",
       "         [-0.9983,  0.0580, -0.0420],\n",
       "         [-0.9978,  0.0662,  0.1736],\n",
       "         [-0.9997,  0.0264,  0.1888],\n",
       "         [-0.9983,  0.0577, -0.0362],\n",
       "         [-0.9993, -0.0369,  0.1911],\n",
       "         [-0.9981,  0.0614,  0.0785],\n",
       "         [-0.9987, -0.0516,  0.1296],\n",
       "         [-0.9997,  0.0224,  0.2712],\n",
       "         [-0.9988,  0.0491, -0.0460],\n",
       "         [-0.9985, -0.0538, -0.3919],\n",
       "         [-1.0000, -0.0067,  0.2941],\n",
       "         [-0.9988,  0.0488, -0.0390],\n",
       "         [-0.9988,  0.0489, -0.0024],\n",
       "         [-0.9986,  0.0535, -0.0071],\n",
       "         [-1.0000,  0.0030,  0.2770],\n",
       "         [-1.0000,  0.0026, -0.2532],\n",
       "         [-0.9973,  0.0735, -0.1833],\n",
       "         [-0.9974,  0.0721, -0.0330],\n",
       "         [-0.9991,  0.0429,  0.1082],\n",
       "         [-0.9987,  0.0508, -0.0851],\n",
       "         [-0.9989,  0.0465, -0.1571],\n",
       "         [-0.9996, -0.0293, -0.4923],\n",
       "         [-0.9993,  0.0386,  0.1025],\n",
       "         [-0.9997, -0.0224,  0.1963],\n",
       "         [-1.0000,  0.0017, -0.3459],\n",
       "         [-0.9974, -0.0723,  0.0049],\n",
       "         [-0.9992,  0.0393, -0.3006],\n",
       "         [-0.9974, -0.0720,  0.1190],\n",
       "         [-0.9890, -0.1476, -0.2848]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0]]))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取一批数据样本\n",
    "def get_sample():\n",
    "    #从样本池中采样\n",
    "    samples = random.sample(datas, 64)\n",
    "\n",
    "    #[b, 3]\n",
    "    state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    action = torch.LongTensor([i[1] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    reward = torch.FloatTensor([i[2] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 3]\n",
    "    next_state = torch.FloatTensor([i[3] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    over = torch.LongTensor([i[4] for i in samples]).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "state, action, reward, next_state, over"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.1331],\n",
       "        [0.2707],\n",
       "        [0.2699],\n",
       "        [0.2643],\n",
       "        [0.1342],\n",
       "        [0.1983],\n",
       "        [0.2801],\n",
       "        [0.1916],\n",
       "        [0.2124],\n",
       "        [0.1361],\n",
       "        [0.1357],\n",
       "        [0.2736],\n",
       "        [0.2580],\n",
       "        [0.2348],\n",
       "        [0.2176],\n",
       "        [0.1973],\n",
       "        [0.2857],\n",
       "        [0.2372],\n",
       "        [0.2070],\n",
       "        [0.1988],\n",
       "        [0.2068],\n",
       "        [0.1602],\n",
       "        [0.1418],\n",
       "        [0.1816],\n",
       "        [0.1411],\n",
       "        [0.2607],\n",
       "        [0.1776],\n",
       "        [0.2082],\n",
       "        [0.2293],\n",
       "        [0.2855],\n",
       "        [0.2638],\n",
       "        [0.1682],\n",
       "        [0.0303],\n",
       "        [0.2334],\n",
       "        [0.2983],\n",
       "        [0.1885],\n",
       "        [0.2419],\n",
       "        [0.2563],\n",
       "        [0.1897],\n",
       "        [0.2697],\n",
       "        [0.2160],\n",
       "        [0.2558],\n",
       "        [0.2795],\n",
       "        [0.1894],\n",
       "        [0.1713],\n",
       "        [0.2916],\n",
       "        [0.0261],\n",
       "        [0.1990],\n",
       "        [0.1971],\n",
       "        [0.2850],\n",
       "        [0.1776],\n",
       "        [0.1570],\n",
       "        [0.1876],\n",
       "        [0.2291],\n",
       "        [0.1814],\n",
       "        [0.1672],\n",
       "        [0.1392],\n",
       "        [0.2285],\n",
       "        [0.2685],\n",
       "        [0.1589],\n",
       "        [0.2496],\n",
       "        [0.1522],\n",
       "        [0.2566],\n",
       "        [0.2245]], grad_fn=<GatherBackward0>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_value(state, action):\n",
    "    #使用状态计算出动作的logits\n",
    "    #[b, 3] -> [b, 11]\n",
    "    value = model(state)\n",
    "\n",
    "    #根据实际使用的action取出每一个值\n",
    "    #这个值就是模型评估的在该状态下,执行动作的分数\n",
    "    #在执行动作前,显然并不知道会得到的反馈和next_state\n",
    "    #所以这里不能也不需要考虑next_state和reward\n",
    "    #[b, 11] -> [b, 1]\n",
    "    value = value.gather(dim=1, index=action)\n",
    "\n",
    "    return value\n",
    "\n",
    "\n",
    "get_value(state, action)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-9.5052],\n",
       "        [-9.4988],\n",
       "        [-9.5066],\n",
       "        [-9.4972],\n",
       "        [-9.6605],\n",
       "        [-9.3041],\n",
       "        [-9.3768],\n",
       "        [-9.2617],\n",
       "        [-9.3816],\n",
       "        [-9.6672],\n",
       "        [-9.7224],\n",
       "        [-9.2520],\n",
       "        [-9.4413],\n",
       "        [-9.3153],\n",
       "        [-9.3217],\n",
       "        [-9.3612],\n",
       "        [-9.4488],\n",
       "        [-9.1665],\n",
       "        [-9.1318],\n",
       "        [-9.2986],\n",
       "        [-9.3227],\n",
       "        [-9.3948],\n",
       "        [-9.6770],\n",
       "        [-9.2958],\n",
       "        [-9.6856],\n",
       "        [-9.4276],\n",
       "        [-9.2756],\n",
       "        [-9.2778],\n",
       "        [-9.2113],\n",
       "        [-9.2945],\n",
       "        [-9.3919],\n",
       "        [-9.3286],\n",
       "        [-9.2966],\n",
       "        [-9.3102],\n",
       "        [-9.4606],\n",
       "        [-9.3279],\n",
       "        [-9.1518],\n",
       "        [-9.3894],\n",
       "        [-9.3260],\n",
       "        [-9.4444],\n",
       "        [-9.2374],\n",
       "        [-9.3478],\n",
       "        [-9.3694],\n",
       "        [-9.3846],\n",
       "        [-9.2848],\n",
       "        [-9.5407],\n",
       "        [-9.3875],\n",
       "        [-9.3615],\n",
       "        [-9.3357],\n",
       "        [-9.4885],\n",
       "        [-9.6544],\n",
       "        [-9.3105],\n",
       "        [-9.2355],\n",
       "        [-9.3337],\n",
       "        [-9.3957],\n",
       "        [-9.4615],\n",
       "        [-9.4123],\n",
       "        [-9.3634],\n",
       "        [-9.5343],\n",
       "        [-9.6461],\n",
       "        [-9.1953],\n",
       "        [-9.5859],\n",
       "        [-9.2166],\n",
       "        [-8.6957]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_target(reward, next_state, over):\n",
    "    #上面已经把模型认为的状态下执行动作的分数给评估出来了\n",
    "    #下面使用next_state和reward计算真实的分数\n",
    "    #针对一个状态,它到底应该多少分,可以使用以往模型积累的经验评估\n",
    "    #这也是没办法的办法,因为显然没有精确解,这里使用延迟更新的next_model评估\n",
    "\n",
    "    #使用next_state计算下一个状态的分数\n",
    "    #[b, 3] -> [b, 11]\n",
    "    with torch.no_grad():\n",
    "        target = next_model(next_state)\n",
    "\n",
    "    #取所有动作中分数最大的\n",
    "    #[b, 11] -> [b, 1]\n",
    "    target = target.max(dim=1)[0]\n",
    "    target = target.reshape(-1, 1)\n",
    "\n",
    "    #下一个状态的分数乘以一个系数,相当于权重\n",
    "    target *= 0.98\n",
    "\n",
    "    #如果next_state已经游戏结束,则next_state的分数是0\n",
    "    #因为如果下一步已经游戏结束,显然不需要再继续玩下去,也就不需要考虑next_state了.\n",
    "    #[b, 1] * [b, 1] -> [b, 1]\n",
    "    target *= (1 - over)\n",
    "\n",
    "    #加上reward就是最终的分数\n",
    "    #[b, 1] + [b, 1] -> [b, 1]\n",
    "    target += reward\n",
    "\n",
    "    return target\n",
    "\n",
    "\n",
    "get_target(reward, next_state, over)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1877.8849049378914"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        _, action_continuous = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step([action_continuous])\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play and random.random() < 0.2:  #跳帧\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "OHoSU6uI-xIt",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 400 200 0 -1351.7574837998905\n",
      "20 4400 200 0 -854.8859494352442\n",
      "40 5000 200 200 -1080.6738543512997\n",
      "60 5000 200 200 -500.39668130791813\n",
      "80 5000 200 200 -490.3536342185761\n",
      "100 5000 200 200 -477.3427797512142\n",
      "120 5000 200 200 -347.5999684421429\n",
      "140 5000 200 200 -122.2401936847602\n",
      "160 5000 200 200 -250.340583666468\n",
      "180 5000 200 200 -291.1251972788201\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(200):\n",
    "        #更新N条数据\n",
    "        update_count, drop_count = update_data()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(200):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "            #计算一批样本的value和target\n",
    "            value = get_value(state, action)\n",
    "            target = get_target(reward, next_state, over)\n",
    "\n",
    "            #更新参数\n",
    "            loss = loss_fn(value, target)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            #把model的参数复制给next_model\n",
    "            if (i + 1) % 50 == 0:\n",
    "                next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 20 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(20)]) / 20\n",
    "            print(epoch, len(datas), update_count, drop_count, test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAD8CAYAAAB3lxGOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQSElEQVR4nO3dX4xU533G8e8zw/4D7BjMgoHFZttuJOO4dawVdeXKsRxX4DgKFhISUVJxgYQUUclRK6XQKKki1ZLbiygXkS9QYnWjpEFIsWRkOWkRSRxFrYyXGLtgjNkEAmuIdx1i82dh/8z+erEn7pidZYedmZ0Zv89HGs0577wz+4zYfThz5swZRQRmlq5cvQOYWX25BMwS5xIwS5xLwCxxLgGzxLkEzBJXsxKQtFHSCUkDknbV6ueYWWVUi+MEJOWBt4C/AQaBV4DPR8QbVf9hZlaRWm0JrAcGIuI3ETEG7AU21ehnmVkFFtTocVcDZ4vWB4G/nGnysmXLYu3atTWKYmYAhw8ffjciOq8fr1UJqMTYh153SNoB7AC488476e/vr1EUMwOQ9NtS47V6OTAIrCla7wLOFU+IiD0R0RsRvZ2d08rJzOZJrUrgFaBHUrekVmArsL9GP8vMKlCTlwMRMSHp74D/BPLAsxFxrBY/y8wqU6t9AkTEi8CLtXp8M6sOHzFoljiXgFniXAJmiXMJmCXOJWCWOJeAWeJcAmaJcwmYJc4lYJY4l4BZ4lwCZolzCZglziVgljiXgFniXAJmiXMJmCXOJWCWOJeAWeJcAmaJcwmYJc4lYJY4l4BZ4lwCZolzCZglziVgljiXgFniXAJmiXMJmCXOJWCWOJeAWeJcAmaJcwmYJc4lYJY4l4BZ4lwCZombtQQkPStpSNLRorGlkg5IOpldLym6bbekAUknJG2oVXAzq45ytgT+Hdh43dgu4GBE9AAHs3UkrQO2Avdk93lGUr5qac2s6mYtgYj4BXDhuuFNQF+23Ac8UTS+NyJGI+IUMACsr05UM6uFue4TWBER5wGy6+XZ+GrgbNG8wWxsGkk7JPVL6h8eHp5jDDOrVLV3DKrEWJSaGBF7IqI3Ino7OzurHMPMyjXXEnhH0kqA7HooGx8E1hTN6wLOzT2emdXaXEtgP7AtW94GPF80vlVSm6RuoAc4VFlEM6ulBbNNkPRD4GFgmaRB4J+Bp4F9krYDZ4AtABFxTNI+4A1gAtgZEYUaZTezKpi1BCLi8zPc9OkZ5j8FPFVJKDObPz5i0CxxLgGzxLkEzBLnEjBL3Kw7Bi1dEcH4H/7A5aNHGf3d78i1t7P47rvpWLuWXEtLveNZlbgErKSYnOT9V17h7e99j2vnzkGhABL5jg6WPvwwq77wBfKLFyOVOkjUmolLwKaJCC4eOcJvv/1tJt5/v/gGCiMjDP/4xxSuXeOuL30JtbXVL6hVhfcJ2DSFK1d4u6/vwwVQLIILL73Eey+/TETJj4ZYE3EJ2DQjAwNcPX36xpMKBYZffJEYH5+XTFY7LgGbJgoFKON/+Mtvvjl7WVjDcwnY3EUwcflyvVNYhVwCVpHCyEi9I1iFXAI2dxGM//739U5hFXIJ2DT5jg5U5sFA3ifQ/FwCNk3r8uUs+NjH6h3D5olLwKbJtbaiBT6OLBUuAZsm19ZGrswSiELBBww1OZeATaN8HnLl/WpMjo5OHVdgTcslYBUpXL069eEia1ouAavIxMWLTPrQ4abmErDpJFrKfHdg9Px5Jq9dq3EgqyWXgE0n0dHdXe8UNk9cAjadRH7RonqnsHniErCS8gsXlj03JidrmMRqzSVgJeVaW8ubODnJ5NWrtQ1jNeUSsGlu5ryBkZ1yzJqXS8AqMznJ+IUL9U5hFXAJWEn5W26BfH7WeTExwbXBwXlIZLXiErCS2letIuczCSfBJWAl5drbUZmfHwD8IaIm5hKwkvIdHWWXgN8ibG4uAStJLS1Q5rsEhatXyzo7sTUml4DNrMwSmBwZcQk0MZeAVawwMuJ9Ak1s1hKQtEbSzyQdl3RM0pPZ+FJJBySdzK6XFN1nt6QBSSckbajlE7DaUC7HgltvLWvuyKlTxMREjRNZrZSzJTAB/ENE3A08AOyUtA7YBRyMiB7gYLZOdttW4B5gI/CMpNnfcLaGopYW2levLmtujI3VOI3V0qwlEBHnI+JX2fIl4DiwGtgE9GXT+oAnsuVNwN6IGI2IU8AAsL7Kua3GlMuR6+iodwybBze1T0DSWuCTwMvAiog4D1NFASzPpq0GzhbdbTAbs2aSy5G/mRLwPoGmVXYJSFoM/Aj4ckRcvNHUEmPTfkMk7ZDUL6l/eHi43Bg2j3JlfgFJFApMjo7WOI3VSlklIKmFqQL4QUQ8lw2/I2lldvtKYCgbHwTWFN29Czh3/WNGxJ6I6I2I3s7OzrnmtxqRVPZbhFEo+BRjTaycdwcEfBc4HhHfLLppP7AtW94GPF80vlVSm6RuoAc4VL3I1miiUKDgEmha5XzDxIPA3wL/K+lINvZPwNPAPknbgTPAFoCIOCZpH/AGU+8s7IwIn5O6CS247baprYFZXu8XrlxhdHCQhWvXzksuq65ZSyAifknp1/kAn57hPk8BT1WQyxpAx5o1KJeb/ctFIpj0cQJNy0cM2oxyCxeWvV/AmpdLwGaUv5kSiPChw03KJWAzKvtko2SfJLSm5BKwqvDJRpuXS8CqonDpUr0j2By5BGxGudZWFtxyS1lzr7z1Vo3TWK24BGxG+YULaVm2rKy5k2Nj/vxAk3IJ2Iy0YAH59vZ6x7AacwnYjJTP+7TjCXAJ2MxyuakTjpYhJiZ8dqEm5RKwGd3MdxJOjo9P7RewpuMSsKqIsTGXQJNyCdgNtZb57sDYu+8y5pPDNKVyPkpsCWvv6vrQ+sWxMZ47c4bha9fYsGoV9y5ZMvWyYXISZvu0oTUkl4DdUH7Rog+WL42P8/VXX+WXQ1MnkfrJ4CD/cv/9/NXy5TPd3ZqAXw7YDRWXwNsjI/z30NAH6++Pj/Nf56adOc6ajEvAbqjjrrtozc4B2ZrL0Zb/8FdI3FrmW4jWuFwCdkMtS5ey5KGHAOhevJh/vPdelrW10ZbP88jKlWzv6QGgdflyWu+4o55RbY68T8BuSBKdjz3GxcOHuXr6NI93dXH/7bdzdWKC1YsW0Z7Po9ZWVmzeTMuSJbM/oDUcbwnYrFo7O7lz5046urtRLseqhQv501tvpT2fJ7dwIXds2cKyRx+9qYOLrHF4S8BmJYlFH/84f/a1r3HhpZe49PrrFK5do2PNGpZ+6lMsuvtucgv8q9Ss/C9nZZFE67JlrNi8mRWbN099bDj7n99bAM3NJWA35YM/eP/hf2R4n4BZ4lwCZolzCZglziVgljiXgFniXAJmiXMJmCXOJWCWOJeAWeJcAmaJcwmYJc4lYJa4WUtAUrukQ5Jek3RM0jey8aWSDkg6mV0vKbrPbkkDkk5I2lDLJ2BmlSlnS2AUeCQi/gK4D9go6QFgF3AwInqAg9k6ktYBW4F7gI3AM5LypR7YzOpv1hKIKZez1ZbsEsAmoC8b7wOeyJY3AXsjYjQiTgEDwPpqhjaz6ilrn4CkvKQjwBBwICJeBlZExHmA7PqPJ59fDZwtuvtgNnb9Y+6Q1C+pf9jfXGNWN2WVQEQUIuI+oAtYL+kTN5he6mwTUeIx90REb0T0dmantDaz+XdT7w5ExHvAz5l6rf+OpJUA2fUfv5ViEFhTdLcuwN9QYdagynl3oFPSbdlyB/Ao8CawH9iWTdsGPJ8t7we2SmqT1A30AIeqnNvMqqSccwyuBPqyPfw5YF9EvCDpf4B9krYDZ4AtABFxTNI+4A1gAtgZEf6mSrMGpYhpL9fnXW9vb/T399c7htlHmqTDEdF7/biPGDRLnEvALHEuAbPEuQTMEucSMEucS8AscS4Bs8S5BMwS5xIwS5xLwCxxLgGzxLkEzBLnEjBLnEvALHEuAbPEuQTMEucSMEucS8AscS4Bs8S5BMwS5xIwS5xLwCxxLgGzxLkEzBLnEjBLnEvALHEuAbPEuQTMEucSMEucS8AscS4Bs8S5BMwS5xIwS5xLwCxxZZeApLykVyW9kK0vlXRA0snseknR3N2SBiSdkLShFsHNrDpuZkvgSeB40fou4GBE9AAHs3UkrQO2AvcAG4FnJOWrE9fMqq2sEpDUBTwOfKdoeBPQly33AU8Uje+NiNGIOAUMAOurktbMqq7cLYFvAV8BJovGVkTEeYDsenk2vho4WzRvMBszswY0awlI+iwwFBGHy3xMlRiLEo+7Q1K/pP7h4eEyH9rMqq2cLYEHgc9JOg3sBR6R9H3gHUkrAbLroWz+ILCm6P5dwLnrHzQi9kREb0T0dnZ2VvAUzKwSs5ZAROyOiK6IWMvUDr+fRsQXgf3AtmzaNuD5bHk/sFVSm6RuoAc4VPXkZlYVCyq479PAPknbgTPAFoCIOCZpH/AGMAHsjIhCxUnNrCYUMe3l+rzr7e2N/v7+escw+0iTdDgieq8f9xGDZolzCZglziVgljiXgFniXAJmiXMJmCXOJWCWOJeAWeJcAmaJcwmYJc4lYJY4l4BZ4lwCZolzCZglziVgljiXgFniXAJmiXMJmCXOJWCWOJeAWeJcAmaJcwmYJc4lYJY4l4BZ4lwCZolzCZglziVgljiXgFniXAJmiXMJmCXOJWCWOJeAWeJcAmaJcwmYJc4lYJY4l4BZ4lwCZolzCZglThFR7wxIGgauAO/WO0uZltE8WaG58jpr7dwVEZ3XDzZECQBI6o+I3nrnKEczZYXmyuus888vB8wS5xIwS1wjlcCeege4Cc2UFZorr7POs4bZJ2Bm9dFIWwJmVgd1LwFJGyWdkDQgaVe98wBIelbSkKSjRWNLJR2QdDK7XlJ02+4s/wlJG+Y56xpJP5N0XNIxSU82al5J7ZIOSXoty/qNRs1a9PPzkl6V9EKjZ52ziKjbBcgDvwb+BGgFXgPW1TNTlush4H7gaNHYvwG7suVdwL9my+uy3G1Ad/Z88vOYdSVwf7Z8C/BWlqnh8gICFmfLLcDLwAONmLUo898D/wG80Mi/B5Vc6r0lsB4YiIjfRMQYsBfYVOdMRMQvgAvXDW8C+rLlPuCJovG9ETEaEaeAAaae17yIiPMR8ats+RJwHFjdiHljyuVstSW7RCNmBZDUBTwOfKdouCGzVqLeJbAaOFu0PpiNNaIVEXEepv7wgOXZeMM8B0lrgU8y9T9sQ+bNNq+PAEPAgYho2KzAt4CvAJNFY42adc7qXQIqMdZsb1c0xHOQtBj4EfDliLh4o6klxuYtb0QUIuI+oAtYL+kTN5het6ySPgsMRcThcu9SYqwpfpfrXQKDwJqi9S7gXJ2yzOYdSSsBsuuhbLzuz0FSC1MF8IOIeC4bbti8ABHxHvBzYCONmfVB4HOSTjP1MvURSd9v0KwVqXcJvAL0SOqW1ApsBfbXOdNM9gPbsuVtwPNF41sltUnqBnqAQ/MVSpKA7wLHI+KbjZxXUqek27LlDuBR4M1GzBoRuyOiKyLWMvV7+dOI+GIjZq1YvfdMAp9hao/2r4Gv1jtPlumHwHlgnKmG3w7cDhwETmbXS4vmfzXLfwJ4bJ6z/jVTm52vA0eyy2caMS/w58CrWdajwNez8YbLel3uh/n/dwcaOutcLj5i0Cxx9X45YGZ15hIwS5xLwCxxLgGzxLkEzBLnEjBLnEvALHEuAbPE/R/YXV5lmP2HugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-114.66159180623404"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第7章-DQN算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
